import redis
from scrapy.dupefilters import RFPDupeFilter  # 默认的去重类
from scrapy.utils.request import request_fingerprint  # 指纹方法


class CustomRFPDupeFilter(RFPDupeFilter):  # 定义去重类，继承自默认去重类
    def __init__(self, settings, path=None, debug=False):
        pool = redis.ConnectionPool(host=settings.get('REDISHOST'), port=settings.get('REDISHOST'),
                                    db=settings.get('REDISHOST'))
        self.rd = redis.Redis(connection_pool=pool)
        super().__init__(path=path, debug=debug)

    def request_seen(self, request):
        fp = request_fingerprint(request)
        if self.rd.sismember('fp', fp):
            return True
        self.rd.sadd('fp', fp)

    @classmethod
    def remove_fp(cls, fp, settings):
        print(fp, '&&&&&&&&&&&&&&&&')
        cls(settings).rd.srem('fp', fp)


class CustomRedisSpiderMiddleware:
    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.
        if response.status > 400:
            fp = request_fingerprint(request)
            CustomRFPDupeFilter.remove_fp(fp, spider.settings)
        return response
